!! Copyright (C) 2009,2010,2011,2012  Marco Restelli
!!
!! This file is part of:
!!   FEMilaro -- Finite Element Method toolkit
!!
!! FEMilaro is free software; you can redistribute it and/or modify it
!! under the terms of the GNU General Public License as published by
!! the Free Software Foundation; either version 3 of the License, or
!! (at your option) any later version.
!!
!! FEMilaro is distributed in the hope that it will be useful, but
!! WITHOUT ANY WARRANTY; without even the implied warranty of
!! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
!! General Public License for more details.
!!
!! You should have received a copy of the GNU General Public License
!! along with FEMilaro; If not, see <http://www.gnu.org/licenses/>.
!!
!! author: Marco Restelli                   <marco.restelli@gmail.com>

!> SeLaLib interface to MPI.
!!
!! See \c sll_collective for additional details.
!<----------------------------------------------------------------------
module mod_mpi_utils

!-----------------------------------------------------------------------

 use mod_messages, only: &
   mod_messages_initialized, &
   error,   &
   warning, &
   info

 use mod_kinds, only: &
   mod_kinds_initialized, &
   wp_p, wp_r

 ! This is the SeLaLib interface to MPI: notice that this module has
 ! "use mpi" and is completely public, so we get access to the whole
 ! MPI interfaces.
 use sll_collective

!-----------------------------------------------------------------------

 implicit none

!-----------------------------------------------------------------------

! Module interface

 public :: &
   mod_mpi_utils_constructor, &
   mod_mpi_utils_destructor,  &
   mod_mpi_utils_initialized, &
   mpi_logical, mpi_integer, wp_mpi, &
   mpi_comm_world, mpi_status_size,  &
   mpi_undefined, mpi_comm_split,    &
   mpi_comm_free,                    &
   mpi_sum, mpi_max, mpi_lor,        &
   mpi_init, mpi_finalize,           &
   mpi_comm_size, mpi_comm_rank,     &
   mpi_barrier,                      &
   mpi_bcast,                        &
   mpi_isend, mpi_irecv,             &
   mpi_send,  mpi_recv,              &
   mpi_request_null,                 &
   mpi_wait, mpi_waitall,            &
   mpi_sendrecv,                     &
   mpi_gather, mpi_allgather,        &
   mpi_reduce, mpi_allreduce,        &
   mpi_alltoall, mpi_alltoallv!,      &
   !mpix_iallreduce

 private

!-----------------------------------------------------------------------

 ! These are the functions that are not defined in the module "mpi"
 external :: mpi_finalize, mpi_isend, mpi_irecv, mpi_send, mpi_recv,   &
   mpi_alltoall, mpi_alltoallv, mpi_bcast, mpi_reduce, mpi_allreduce,  &
   mpi_sendrecv, mpi_waitall, mpi_gather, mpi_allgather
   !mpix_iallreduce,                                                    &

! Module variables
 ! public members
 integer, protected :: wp_mpi !< MPI representation of wp
 logical, protected ::               &
   mod_mpi_utils_initialized = .false.
 ! private members
 logical :: already_initialized_mpi
 character(len=*), parameter :: &
   this_mod_name = 'mod_mpi_utils'

!-----------------------------------------------------------------------

contains

!-----------------------------------------------------------------------

 !> Setup the sll MPI interface.
 !!
 !! In FEMilaro, MPI must be explicitly initialized before calling
 !! this function, while in SeLaLib a call to \c mpi_init is included
 !! in \c sll_boot_collective. To handle this, we first call \c
 !! mpi_finalize (if necessary) and then reinitialize MPI through the
 !! SeLaLib interface. If MPI is stopped, then it is also restarted
 !! after calling \c sll_halt_collective.
 subroutine mod_mpi_utils_constructor()

  integer :: ierr
  character(len=100) message
  character(len=*), parameter :: &
    this_sub_name = 'constructor'

   !Consistency checks ---------------------------
   if( (mod_messages_initialized.eqv..false.) .or. &
          (mod_kinds_initialized.eqv..false.) ) then
     call error(this_sub_name,this_mod_name, &
                'Not all the required modules are initialized.')
   endif
   if(mod_mpi_utils_initialized.eqv..true.) then
     call warning(this_sub_name,this_mod_name, &
                  'Module is already initialized.')
   endif

   call mpi_initialized(already_initialized_mpi,ierr)
   if(already_initialized_mpi) call mpi_finalize(ierr)

   call sll_boot_collective()

   !----------------------------------------------

   ! Compiler bug: mpich2 has problems with mpi_type_create_f90_real,
   ! see https://trac.mcs.anl.gov/projects/mpich2/ticket/1028 . This
   ! should be fixed, so we use here the "nice" way - however, if you
   ! see problems here you should select manually the working
   ! precision uncommenting the corresponding line.
   !wp_mpi = mpi_double_precision
   !wp_mpi = mpi_real16
   call mpi_type_create_f90_real(wp_p,wp_r,wp_mpi,ierr)

   write(message,*) 'mpi representation of wp = ', wp_mpi
   call info(this_sub_name,this_mod_name,message)

   mod_mpi_utils_initialized = .true.
 end subroutine mod_mpi_utils_constructor

!-----------------------------------------------------------------------
 
 subroutine mod_mpi_utils_destructor()
  character(len=*), parameter :: &
    this_sub_name = 'destructor'
   
  integer :: ierr

   !Consistency checks ---------------------------
   if(mod_mpi_utils_initialized.eqv..false.) then
     call error(this_sub_name,this_mod_name, &
                'This module is not initialized.')
   endif
   !----------------------------------------------

   call sll_halt_collective()

   if(already_initialized_mpi) call mpi_init(ierr)

   mod_mpi_utils_initialized = .false.
 end subroutine mod_mpi_utils_destructor

!-----------------------------------------------------------------------

end module mod_mpi_utils

